var path = require('path'), request = require('request'), cheerio = require('cheerio');
var Crawler = require('../../lib/crawler');
//var connection = 'mongodb://localhost:27017/66xiaoqu';
//var connection = 'mongodb://192.168.1.123:27017/66xiaoqu';
var connection = 'mongodb://120.55.126.201:27017/66xiaoqu';
var async = require('async');
var MongoClient = require('mongodb').MongoClient, assert = require('assert');

if (!String.prototype.format) {
    String.prototype.format = function () {
        var args = arguments;
        return this.replace(/{(\d+)}/g, function (match, number) {
            return typeof args[number] != 'undefined'
                ? args[number]
                : match
                ;
        });
    };
}
var filteredWords = '小区,区,街,路,市,县,苑,花苑,花园,园,属,院,家,楼,巷,公寓,公房,大厦,大楼,别墅'.split(',')
function filterAddress(addr) {
    var new_addr = addr
    filteredWords.forEach(function (word) {
        new_addr = new_addr.replace(word, '')
        //console.log(word)
    })
    return new_addr
}

var dbconn = null;

function update() {
    var urlTemp = 'http://api.map.baidu.com/place/v2/search?&query={0}&address={1}&region={2}&tag=%E5%B0%8F%E5%8C%BA&scope=2&output=json&ak=AvRqTglpaKICTZ5iwyn06KZj&page_size=1&page_num=0'
    //var urlTemp = 'http://api.map.baidu.com/place/v2/search?&query={0}&address={1}&region={2}&tag=%E5%B0%8F%E5%8C%BA&scope=2&output=json&ak=doqfhHvA5GjNLYnjjVFm9sL8&page_size=1&page_num=0'
    //var urlTemp = 'http://api.map.baidu.com/place/v2/search?&query={0}&address={1}&region={2}&tag=%E5%B0%8F%E5%8C%BA&scope=2&output=json&ak=ayGOiNeQNq6E9vSrbgeTrWbS&page_size=1&page_num=

    var dbCollection = 'communityFINAL';
    var collection = null;

    var totalData = 0;
    var counter = 0;

    //var filters = {}
    var filters = {'baidu2':{$exists:false},'location.longitude': {$in: [0, null]}, 'location.latitude': {$in: [0, null]}}

    function getExistingDataCount(callback) {
        MongoClient.connect(connection,
            function (err, db) {
                dbconn = db
                assert.equal(null, err);
                //console.log("Connected correctly to server");
                collection = db.collection(dbCollection);
                collection.count(filters, function (err, count) {
                    totalData = count
                    console.log("Total data %s: %s", dbCollection, totalData)
                    if (totalData == 0)
                        dbconn.close()
                    else
                        callback()
                });
            });
    };

    function run() {
        var manifest = {
            target: {
                url: urlTemp,
                ranges: {
                    resource: 'mongodb' // configure it and put {{resource:mongodb}}
                },
                interval: 100 // Wait 0.5 seconds between each access
            },
            extractors: [{
                name: 'details',
                root: '',
                //root:'',
                fields: {}
            }]
        };

        function insertDocuments(doc, new_doc, callback) {
            var updateSpecs = {'baidu2': true}
            if ('name' in new_doc && filterAddress(doc.communityName) == filterAddress(new_doc.name)) {
                if ('price' in new_doc && (doc.price == 0 || isNaN(doc.price))) {
                    updateSpecs['price'] = parseFloat(new_doc['price'])
                }
                updateSpecs['location'] = {}
                updateSpecs['location']['latitude'] = new_doc.latitude
                updateSpecs['location']['longitude'] = new_doc.longitude
                updateSpecs['baiduAddress'] = new_doc.address
            } else {
                console.log("No data for %s (commId: %s) %s", doc.communityName, doc.commId, dbCollection);
            }
            collection.updateOne({'_id': doc._id}, {$set: updateSpecs}, function (err, result) {
                if (!err) {
                    console.log("Updated %s (commId: %s) %s", doc.communityName, doc.commId, dbCollection);
                    console.log(updateSpecs)
                } else {
                    console.log(err)
                }
                callback();
            });
        };
        var job = new Crawler(manifest);

        job.on('data', function (data) {
        });

        job.on('error', function (err) {
            throw error;
        });

        job.on('end', function () {
            //console.log('The job is done');
            //dbconn.close()
            // not exactly,
        });

        function init(callback) {
            //MongoClient.connect(connection,
            //    function (err, db) {
            //        assert.equal(null, err);
            //        //console.log("Connected correctly to server");
            //        dbconn = db;
            //        collection = db.collection(dbCollection)
            callback(null, 'init');
            //    });
        }

        var ids = [];


        function findCommids(db, callback) {
            collection.find(filters).toArray(
                function (err, docs) {

//            console. log(docs);
                    callback(docs);
                });
        };

        function getIds(callback) {
            findCommids(dbconn, function (docs) {
                    ids = docs;
                    callback(null, 'getIds');
                }
            );
        }

        function start(callback) {
            console.log("starting...");
            var cnt = 0;

            async.eachSeries(ids, function (item, callbackInner) {
                    counter++
                    console.log("(%s of %s)", counter, totalData)

                    var query = ''
                    var queryAddress = ''
                    if(item.source == 'anjuke'){
                        query = item.city + ' ' + item.district + ' ' + item.communityName
                        queryAddress = item.communityAddress.substr(item.communityAddress.indexOf(']') + 1)
                    }else if(item.source == 'sofang'){
                        query = item.city + ' ' + item.communityName
                        queryAddress = item.communityAddress
                    }else{
                        callbackInner()
                        return
                    }
                    manifest.target.url = urlTemp.format(encodeURIComponent(query), encodeURIComponent(queryAddress), encodeURIComponent((item.city)))
                    //console.log(manifest.target.url)

                    var job1 = new Crawler(manifest);
                    //console.log(job1)

                    job1.on('data', function (data) {
                        //console.log(data)
                        var $ = cheerio.load(data.request.body);
                        var dataRoot = JSON.parse($._root.children[0].data)
                        //console.log(dataRoot)
                        if (parseInt(dataRoot.status) == 302) {
                            console.log("YOU HAVE BEEN BLOCKED BY BAIDU!")
                            process.exit()
                        }
                        else if (parseInt(dataRoot.status) == 240) {
                            console.log("INVALID DEVELOPER KEY!")
                            process.exit()
                        }
                        else if (parseInt(dataRoot.status) == 210) {
                            console.log("IP NOT ALLOWED FOR THIS DEVELOPER KEY!")
                            process.exit()
                        }

                        var new_doc = {}
                        try {
                            var data = dataRoot.results
                            var doc = data[0]  // only most relevant needed
                            for (var f in doc) {
                                if (f == 'location') {
                                    new_doc['longitude'] = doc[f]['lng']
                                    new_doc['latitude'] = doc[f]['lat']
                                } else if (f == 'detail_info') {
                                    for (var df in doc[f]) {
                                        new_doc[df] = doc[f][df]
                                    }
                                } else {
                                    new_doc[f] = doc[f]
                                }
                            }
                        } catch (e) {
                            console.log(e)
                        }
                        insertDocuments(item, new_doc, function () {
                            //callback()
                            if (counter == totalData) {
                                setTimeout(function () {
                                    dbconn.close()
                                }, 1000)
                            }
                        })

                    });

                    job1.on('error', function (err) {
                        throw err;
                    });

                    job1.on('end', function () {

                        console.log('The job is done: %d', ++cnt);
                        delete job1;

                        callbackInner();

                    });

                    job1.start();

                }, function (err) {
                    if (err) {
                        console.log('err');
                    }
                }
            );
        }

        function finish(callback) {
            //console.log("closing db...");
            callback(null, 'end');
        }

        // Start our job
        async.series([
                init,
                getIds,
                start,
                finish
            ]
            , function (err, results) {
                //console.log(results);
            });
    }

    getExistingDataCount(run);
}

update()
